{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 作业：使用LDA模型破解希拉里“邮件门”的秘密\n",
    "作者：加号、助教-Noah"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 任务描述：\n",
    "基于课程中的LDA模型，判断希拉里Twitter中的每句话分别属于哪个Topic<br>\n",
    "\n",
    "1. 构建LDA模型<br>\n",
    "2. 任务一：读取数据<br>\n",
    "3. 任务二：数据预处理<br>\n",
    "4. 任务三：词袋化<br>\n",
    "5. 任务四：判断主题<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. 创建LDA模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.007*\"percent\" + 0.007*\"would\" + 0.006*\"today\" + 0.006*\"discuss\" + 0.005*\"heard\"'),\n",
       " (1,\n",
       "  '0.008*\"state\" + 0.007*\"secretary\" + 0.007*\"us\" + 0.006*\"federal\" + 0.004*\"department\"'),\n",
       " (2,\n",
       "  '0.008*\"said\" + 0.008*\"would\" + 0.008*\"party\" + 0.006*\"bill\" + 0.004*\"could\"'),\n",
       " (3,\n",
       "  '0.007*\"negotiating\" + 0.007*\"original\" + 0.007*\"verveer\" + 0.006*\"wing\" + 0.006*\"camps\"'),\n",
       " (4,\n",
       "  '0.028*\"pm\" + 0.008*\"huma\" + 0.007*\"office\" + 0.007*\"abedin\" + 0.006*\"secretary\"'),\n",
       " (5,\n",
       "  '0.010*\"pm\" + 0.009*\"office\" + 0.007*\"please\" + 0.006*\"beck\" + 0.005*\"cdm\"'),\n",
       " (6,\n",
       "  '0.018*\"state\" + 0.018*\"taliban\" + 0.012*\"secretary\" + 0.011*\"assistant\" + 0.009*\"lona\"'),\n",
       " (7,\n",
       "  '0.011*\"part\" + 0.010*\"release\" + 0.008*\"think\" + 0.007*\"know\" + 0.006*\"im\"'),\n",
       " (8,\n",
       "  '0.022*\"bloomberg\" + 0.010*\"print\" + 0.010*\"pis\" + 0.006*\"thanks\" + 0.006*\"burns\"'),\n",
       " (9, '0.007*\"would\" + 0.006*\"got\" + 0.006*\"thx\" + 0.006*\"done\" + 0.006*\"get\"'),\n",
       " (10,\n",
       "  '0.005*\"one\" + 0.005*\"president\" + 0.004*\"think\" + 0.004*\"new\" + 0.004*\"people\"'),\n",
       " (11,\n",
       "  '0.016*\"state\" + 0.014*\"doc\" + 0.014*\"us\" + 0.008*\"press\" + 0.008*\"strategic\"'),\n",
       " (12,\n",
       "  '0.066*\"fyi\" + 0.010*\"voters\" + 0.009*\"election\" + 0.007*\"palin\" + 0.006*\"vote\"'),\n",
       " (13, '0.032*\"ok\" + 0.020*\"pls\" + 0.013*\"print\" + 0.008*\"call\" + 0.005*\"wjc\"'),\n",
       " (14,\n",
       "  '0.007*\"new\" + 0.007*\"us\" + 0.006*\"obama\" + 0.005*\"one\" + 0.005*\"israel\"'),\n",
       " (15,\n",
       "  '0.082*\"pm\" + 0.040*\"office\" + 0.034*\"secretarys\" + 0.025*\"meeting\" + 0.023*\"room\"'),\n",
       " (16,\n",
       "  '0.011*\"un\" + 0.007*\"women\" + 0.007*\"haitian\" + 0.006*\"us\" + 0.006*\"afghan\"'),\n",
       " (17,\n",
       "  '0.013*\"see\" + 0.011*\"pls\" + 0.011*\"fw\" + 0.009*\"pm\" + 0.009*\"richards\"'),\n",
       " (18,\n",
       "  '0.008*\"would\" + 0.006*\"said\" + 0.006*\"us\" + 0.004*\"nuclear\" + 0.004*\"could\"'),\n",
       " (19,\n",
       "  '0.023*\"call\" + 0.012*\"yes\" + 0.008*\"good\" + 0.008*\"like\" + 0.007*\"see\"')]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 请勿修改次单元格\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "\n",
    "df = pd.read_csv(\"./input/HillaryEmails.csv\")\n",
    "# 原邮件数据中有很多Nan的值，直接扔了。\n",
    "df = df[['Id','ExtractedBodyText']].dropna()\n",
    "\n",
    "### 文本预处理\n",
    "\n",
    "def clean_email_text(text):\n",
    "    text = text.replace('\\n',\" \") #新行，我们是不需要的\n",
    "    text = re.sub(r\"-\", \" \", text) #把 \"-\" 的两个单词，分开。（比如：pre-processing ==> pre processing）\n",
    "    text = re.sub(r\"\\d+/\\d+/\\d+\", \"\", text) #日期，对主体模型没什么意义\n",
    "    text = re.sub(r\"[0-2]?[0-9]:[0-6][0-9]\", \"\", text) #时间，没意义\n",
    "    text = re.sub(r\"[\\w]+@[\\.\\w]+\", \"\", text) #邮件地址，没意义\n",
    "    text = re.sub(r\"/[a-zA-Z]*[:\\//\\]*[A-Za-z0-9\\-_]+\\.+[A-Za-z0-9\\.\\/%&=\\?\\-_]+/i\", \"\", text) #网址，没意义\n",
    "    pure_text = ''\n",
    "    # 以防还有其他特殊字符（数字）等等，我们直接把他们loop一遍，过滤掉\n",
    "    for letter in text:\n",
    "        # 只留下字母和空格\n",
    "        if letter.isalpha() or letter==' ':\n",
    "            pure_text += letter\n",
    "    # 再把那些去除特殊字符后落单的单词，直接排除。\n",
    "    # 我们就只剩下有意义的单词了。\n",
    "    text = ' '.join(word for word in pure_text.split() if len(word)>1)\n",
    "    return text\n",
    "\n",
    "docs = df['ExtractedBodyText']\n",
    "docs = docs.apply(lambda s: clean_email_text(s))  \n",
    "\n",
    "# 把邮件内容拿出来\n",
    "doclist = docs.values\n",
    "\n",
    "\n",
    "### 构建LDA模型\n",
    "\n",
    "from gensim import corpora, models, similarities\n",
    "import gensim\n",
    "#停止词列表\n",
    "stoplist = ['very', 'ourselves', 'am', 'doesn', 'through', 'me', 'against', 'up', 'just', 'her', 'ours', \n",
    "            'couldn', 'because', 'is', 'isn', 'it', 'only', 'in', 'such', 'too', 'mustn', 'under', 'their', \n",
    "            'if', 'to', 'my', 'himself', 'after', 'why', 'while', 'can', 'each', 'itself', 'his', 'all', 'once', \n",
    "            'herself', 'more', 'our', 'they', 'hasn', 'on', 'ma', 'them', 'its', 'where', 'did', 'll', 'you', \n",
    "            'didn', 'nor', 'as', 'now', 'before', 'those', 'yours', 'from', 'who', 'was', 'm', 'been', 'will', \n",
    "            'into', 'same', 'how', 'some', 'of', 'out', 'with', 's', 'being', 't', 'mightn', 'she', 'again', 'be', \n",
    "            'by', 'shan', 'have', 'yourselves', 'needn', 'and', 'are', 'o', 'these', 'further', 'most', 'yourself', \n",
    "            'having', 'aren', 'here', 'he', 'were', 'but', 'this', 'myself', 'own', 'we', 'so', 'i', 'does', 'both', \n",
    "            'when', 'between', 'd', 'had', 'the', 'y', 'has', 'down', 'off', 'than', 'haven', 'whom', 'wouldn', \n",
    "            'should', 've', 'over', 'themselves', 'few', 'then', 'hadn', 'what', 'until', 'won', 'no', 'about', \n",
    "            'any', 'that', 'for', 'shouldn', 'don', 'do', 'there', 'doing', 'an', 'or', 'ain', 'hers', 'wasn', \n",
    "            'weren', 'above', 'a', 'at', 'your', 'theirs', 'below', 'other', 'not', 're', 'him', 'during', 'which']\n",
    "\n",
    "#人工分词\n",
    "texts = [[word for word in doc.lower().split() if word not in stoplist] for doc in doclist]\n",
    "\n",
    "\n",
    "### 建立语料库\n",
    "\n",
    "dictionary = corpora.Dictionary(texts)\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=20)\n",
    "# 打印出所有主题\n",
    "lda.print_topics(num_topics=20, num_words=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.希拉里twitter主题分类\n",
    "### 任务一：读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 读取数据；将数据的列名定义为：text；并将该列数据存放至名为twitt_docs的变量中\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 任务二：文本预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 利用课上定义的clean_model_text函数，进行数据预处理；处理后的结果命名为：testdata\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 任务三：词袋化(利用email建立的dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 进行人工分词，并生成词袋\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 任务四：判断twitter所属的主题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 判断主题\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 版权归 © 稀牛学院 所有 保留所有权利"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](http://pic1.tsingdataedu.com/%E7%A8%80%E7%89%9B%20x%20%E7%BD%91%E6%98%93.png)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
